{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DepthSeperabelConv2d(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, output_channels, kernel_size, activation = 'relu', **kwargs):\n",
    "        super().__init__()\n",
    "        \n",
    "        \n",
    "        if (activation == 'relu'):\n",
    "            f_activation =  nn.ReLU(inplace=True)\n",
    "            \n",
    "        if (activation == 'swish'):\n",
    "            f_activation =  swish()\n",
    "            \n",
    "        if (activation == 'mish'):\n",
    "            f_activation =  mish()\n",
    "        \n",
    "        self.depthwise = nn.Sequential(\n",
    "            nn.Conv2d(\n",
    "                input_channels,\n",
    "                input_channels,\n",
    "                kernel_size,\n",
    "                groups=input_channels,\n",
    "                **kwargs),\n",
    "            nn.BatchNorm2d(input_channels),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.pointwise = nn.Sequential(\n",
    "            nn.Conv2d(input_channels, output_channels, 1),\n",
    "            nn.BatchNorm2d(output_channels),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.depthwise(x)\n",
    "        x = self.pointwise(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "\n",
    "class BasicConv2d(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, output_channels, kernel_size, activation = 'relu', **kwargs):\n",
    "\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(\n",
    "            input_channels, output_channels, kernel_size, **kwargs)\n",
    "        self.bn = nn.BatchNorm2d(output_channels)\n",
    "        \n",
    "        if (activation == 'relu'):\n",
    "            self.activation =  nn.ReLU(inplace=True)\n",
    "            \n",
    "        if (activation == 'swish'):\n",
    "            self.activation =  swish()\n",
    "            \n",
    "        if (activation == 'mish'):\n",
    "            self.activation =  mish()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = self.bn(x)\n",
    "        x = self.activation(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "\n",
    "class MobileNet(nn.Module):\n",
    "\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        width multipler: The role of the width multiplier α is to thin \n",
    "                         a network uniformly at each layer. For a given \n",
    "                         layer and width multiplier α, the number of \n",
    "                         input channels M becomes αM and the number of \n",
    "                         output channels N becomes αN.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, width_multiplier=1, class_num=100, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        alpha = width_multiplier\n",
    "        self.stem = nn.Sequential(\n",
    "           BasicConv2d(3, int(32 * alpha), 3, padding=1, bias=False, activation = activation),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(32 * alpha),\n",
    "               int(64 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False\n",
    "           )\n",
    "       )\n",
    "        self.conv1 = nn.Sequential(\n",
    "           DepthSeperabelConv2d(\n",
    "               int(64 * alpha),\n",
    "               int(128 * alpha),\n",
    "               3,\n",
    "               stride=2,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(128 * alpha),\n",
    "               int(128 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           )\n",
    "       )\n",
    "        self.conv2 = nn.Sequential(\n",
    "           DepthSeperabelConv2d(\n",
    "               int(128 * alpha),\n",
    "               int(256 * alpha),\n",
    "               3,\n",
    "               stride=2,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(256 * alpha),\n",
    "               int(256 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           )\n",
    "       )\n",
    "        self.conv3 = nn.Sequential(\n",
    "           DepthSeperabelConv2d(\n",
    "               int(256 * alpha),\n",
    "               int(512 * alpha),\n",
    "               3,\n",
    "               stride=2,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "\n",
    "           DepthSeperabelConv2d(\n",
    "               int(512 * alpha),\n",
    "               int(512 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(512 * alpha),\n",
    "               int(512 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(512 * alpha),\n",
    "               int(512 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(512 * alpha),\n",
    "               int(512 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(512 * alpha),\n",
    "               int(512 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           )\n",
    "       )\n",
    "        self.conv4 = nn.Sequential(\n",
    "           DepthSeperabelConv2d(\n",
    "               int(512 * alpha),\n",
    "               int(1024 * alpha),\n",
    "               3,\n",
    "               stride=2,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           ),\n",
    "           DepthSeperabelConv2d(\n",
    "               int(1024 * alpha),\n",
    "               int(1024 * alpha),\n",
    "               3,\n",
    "               padding=1,\n",
    "               bias=False,\n",
    "               activation = activation\n",
    "           )\n",
    "       )\n",
    "        self.fc = nn.Linear(int(1024 * alpha), class_num)\n",
    "        self.avg = nn.AdaptiveAvgPool2d(1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.stem(x)\n",
    "\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = self.conv3(x)\n",
    "        x = self.conv4(x)\n",
    "\n",
    "        x = self.avg(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "def mobilenet(alpha=1, class_num=10, activation = 'relu'):\n",
    "    return MobileNet(alpha, class_num, activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27502797.84it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = mobilenet(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 25.4048.. Average time per step: 0.0650.. Train loss: 1.4982.. Train accuracy: 0.4525.. Top-3 train accuracy: 0.7814.. Test loss: 1.2146.. Test accuracy: 0.5578.. Top-3 test accuracy: 0.8584\n",
      "Epoch 2/100.. Time per epoch: 25.0423.. Average time per step: 0.0640.. Train loss: 1.1457.. Train accuracy: 0.5926.. Top-3 train accuracy: 0.8702.. Test loss: 0.9879.. Test accuracy: 0.6482.. Top-3 test accuracy: 0.9022\n",
      "Epoch 3/100.. Time per epoch: 25.1504.. Average time per step: 0.0643.. Train loss: 0.9767.. Train accuracy: 0.6540.. Top-3 train accuracy: 0.9030.. Test loss: 0.8658.. Test accuracy: 0.6915.. Top-3 test accuracy: 0.9255\n",
      "Epoch 4/100.. Time per epoch: 25.0950.. Average time per step: 0.0642.. Train loss: 0.8664.. Train accuracy: 0.6944.. Top-3 train accuracy: 0.9202.. Test loss: 0.8268.. Test accuracy: 0.7188.. Top-3 test accuracy: 0.9314\n",
      "Epoch 5/100.. Time per epoch: 24.9591.. Average time per step: 0.0638.. Train loss: 0.7849.. Train accuracy: 0.7274.. Top-3 train accuracy: 0.9327.. Test loss: 0.7544.. Test accuracy: 0.7436.. Top-3 test accuracy: 0.9338\n",
      "Epoch 6/100.. Time per epoch: 24.9889.. Average time per step: 0.0639.. Train loss: 0.7194.. Train accuracy: 0.7483.. Top-3 train accuracy: 0.9413.. Test loss: 0.6852.. Test accuracy: 0.7630.. Top-3 test accuracy: 0.9454\n",
      "Epoch 7/100.. Time per epoch: 25.0432.. Average time per step: 0.0640.. Train loss: 0.6725.. Train accuracy: 0.7668.. Top-3 train accuracy: 0.9470.. Test loss: 0.6404.. Test accuracy: 0.7802.. Top-3 test accuracy: 0.9524\n",
      "Epoch 8/100.. Time per epoch: 25.0863.. Average time per step: 0.0642.. Train loss: 0.6246.. Train accuracy: 0.7826.. Top-3 train accuracy: 0.9535.. Test loss: 0.6001.. Test accuracy: 0.7955.. Top-3 test accuracy: 0.9561\n",
      "Epoch 9/100.. Time per epoch: 25.0383.. Average time per step: 0.0640.. Train loss: 0.5864.. Train accuracy: 0.7971.. Top-3 train accuracy: 0.9590.. Test loss: 0.6074.. Test accuracy: 0.7926.. Top-3 test accuracy: 0.9572\n",
      "Epoch 10/100.. Time per epoch: 25.1004.. Average time per step: 0.0642.. Train loss: 0.5539.. Train accuracy: 0.8071.. Top-3 train accuracy: 0.9623.. Test loss: 0.5465.. Test accuracy: 0.8143.. Top-3 test accuracy: 0.9610\n",
      "Epoch 11/100.. Time per epoch: 24.9697.. Average time per step: 0.0639.. Train loss: 0.5274.. Train accuracy: 0.8182.. Top-3 train accuracy: 0.9656.. Test loss: 0.5603.. Test accuracy: 0.8096.. Top-3 test accuracy: 0.9602\n",
      "Epoch 12/100.. Time per epoch: 25.0182.. Average time per step: 0.0640.. Train loss: 0.4975.. Train accuracy: 0.8268.. Top-3 train accuracy: 0.9676.. Test loss: 0.5371.. Test accuracy: 0.8201.. Top-3 test accuracy: 0.9616\n",
      "Epoch 13/100.. Time per epoch: 25.0962.. Average time per step: 0.0642.. Train loss: 0.4739.. Train accuracy: 0.8348.. Top-3 train accuracy: 0.9709.. Test loss: 0.5459.. Test accuracy: 0.8174.. Top-3 test accuracy: 0.9621\n",
      "Epoch 14/100.. Time per epoch: 25.1041.. Average time per step: 0.0642.. Train loss: 0.4486.. Train accuracy: 0.8441.. Top-3 train accuracy: 0.9743.. Test loss: 0.5488.. Test accuracy: 0.8213.. Top-3 test accuracy: 0.9614\n",
      "Epoch 15/100.. Time per epoch: 24.9447.. Average time per step: 0.0638.. Train loss: 0.4250.. Train accuracy: 0.8535.. Top-3 train accuracy: 0.9762.. Test loss: 0.5439.. Test accuracy: 0.8222.. Top-3 test accuracy: 0.9614\n",
      "Epoch 16/100.. Time per epoch: 25.0806.. Average time per step: 0.0641.. Train loss: 0.4088.. Train accuracy: 0.8556.. Top-3 train accuracy: 0.9776.. Test loss: 0.5294.. Test accuracy: 0.8309.. Top-3 test accuracy: 0.9634\n",
      "Epoch 17/100.. Time per epoch: 24.9283.. Average time per step: 0.0638.. Train loss: 0.3876.. Train accuracy: 0.8639.. Top-3 train accuracy: 0.9799.. Test loss: 0.5329.. Test accuracy: 0.8285.. Top-3 test accuracy: 0.9659\n",
      "Epoch 18/100.. Time per epoch: 24.9521.. Average time per step: 0.0638.. Train loss: 0.3671.. Train accuracy: 0.8710.. Top-3 train accuracy: 0.9809.. Test loss: 0.5211.. Test accuracy: 0.8296.. Top-3 test accuracy: 0.9671\n",
      "Epoch 19/100.. Time per epoch: 25.0985.. Average time per step: 0.0642.. Train loss: 0.3494.. Train accuracy: 0.8782.. Top-3 train accuracy: 0.9828.. Test loss: 0.5425.. Test accuracy: 0.8224.. Top-3 test accuracy: 0.9646\n",
      "Epoch 20/100.. Time per epoch: 25.0052.. Average time per step: 0.0640.. Train loss: 0.3355.. Train accuracy: 0.8831.. Top-3 train accuracy: 0.9835.. Test loss: 0.5305.. Test accuracy: 0.8298.. Top-3 test accuracy: 0.9697\n",
      "Epoch 21/100.. Time per epoch: 24.9625.. Average time per step: 0.0638.. Train loss: 0.3231.. Train accuracy: 0.8877.. Top-3 train accuracy: 0.9847.. Test loss: 0.5160.. Test accuracy: 0.8366.. Top-3 test accuracy: 0.9690\n",
      "Epoch 22/100.. Time per epoch: 24.9896.. Average time per step: 0.0639.. Train loss: 0.3047.. Train accuracy: 0.8930.. Top-3 train accuracy: 0.9866.. Test loss: 0.4917.. Test accuracy: 0.8399.. Top-3 test accuracy: 0.9705\n",
      "Epoch 23/100.. Time per epoch: 24.9328.. Average time per step: 0.0638.. Train loss: 0.2962.. Train accuracy: 0.8968.. Top-3 train accuracy: 0.9872.. Test loss: 0.5118.. Test accuracy: 0.8384.. Top-3 test accuracy: 0.9706\n",
      "Epoch 24/100.. Time per epoch: 24.9559.. Average time per step: 0.0638.. Train loss: 0.2787.. Train accuracy: 0.9031.. Top-3 train accuracy: 0.9881.. Test loss: 0.5381.. Test accuracy: 0.8395.. Top-3 test accuracy: 0.9696\n",
      "Epoch 25/100.. Time per epoch: 25.0172.. Average time per step: 0.0640.. Train loss: 0.2719.. Train accuracy: 0.9069.. Top-3 train accuracy: 0.9891.. Test loss: 0.5084.. Test accuracy: 0.8437.. Top-3 test accuracy: 0.9692\n",
      "Epoch 26/100.. Time per epoch: 25.0178.. Average time per step: 0.0640.. Train loss: 0.2592.. Train accuracy: 0.9107.. Top-3 train accuracy: 0.9903.. Test loss: 0.5402.. Test accuracy: 0.8365.. Top-3 test accuracy: 0.9683\n",
      "Epoch 27/100.. Time per epoch: 24.9582.. Average time per step: 0.0638.. Train loss: 0.2498.. Train accuracy: 0.9137.. Top-3 train accuracy: 0.9913.. Test loss: 0.5127.. Test accuracy: 0.8475.. Top-3 test accuracy: 0.9688\n",
      "Epoch 28/100.. Time per epoch: 24.9689.. Average time per step: 0.0639.. Train loss: 0.2345.. Train accuracy: 0.9180.. Top-3 train accuracy: 0.9911.. Test loss: 0.5309.. Test accuracy: 0.8474.. Top-3 test accuracy: 0.9700\n",
      "Epoch 29/100.. Time per epoch: 24.9202.. Average time per step: 0.0637.. Train loss: 0.2269.. Train accuracy: 0.9209.. Top-3 train accuracy: 0.9924.. Test loss: 0.5476.. Test accuracy: 0.8380.. Top-3 test accuracy: 0.9692\n",
      "Epoch 30/100.. Time per epoch: 25.0750.. Average time per step: 0.0641.. Train loss: 0.2192.. Train accuracy: 0.9237.. Top-3 train accuracy: 0.9927.. Test loss: 0.5586.. Test accuracy: 0.8373.. Top-3 test accuracy: 0.9693\n",
      "Epoch 31/100.. Time per epoch: 25.0222.. Average time per step: 0.0640.. Train loss: 0.2119.. Train accuracy: 0.9258.. Top-3 train accuracy: 0.9934.. Test loss: 0.5406.. Test accuracy: 0.8450.. Top-3 test accuracy: 0.9704\n",
      "Epoch 32/100.. Time per epoch: 24.9389.. Average time per step: 0.0638.. Train loss: 0.2038.. Train accuracy: 0.9290.. Top-3 train accuracy: 0.9941.. Test loss: 0.5306.. Test accuracy: 0.8472.. Top-3 test accuracy: 0.9725\n",
      "Epoch 33/100.. Time per epoch: 24.9795.. Average time per step: 0.0639.. Train loss: 0.1995.. Train accuracy: 0.9318.. Top-3 train accuracy: 0.9940.. Test loss: 0.5388.. Test accuracy: 0.8482.. Top-3 test accuracy: 0.9719\n",
      "Epoch 34/100.. Time per epoch: 24.9387.. Average time per step: 0.0638.. Train loss: 0.1862.. Train accuracy: 0.9349.. Top-3 train accuracy: 0.9946.. Test loss: 0.5564.. Test accuracy: 0.8494.. Top-3 test accuracy: 0.9712\n",
      "Epoch 35/100.. Time per epoch: 24.9159.. Average time per step: 0.0637.. Train loss: 0.1894.. Train accuracy: 0.9338.. Top-3 train accuracy: 0.9946.. Test loss: 0.5466.. Test accuracy: 0.8468.. Top-3 test accuracy: 0.9694\n",
      "Epoch 36/100.. Time per epoch: 24.9636.. Average time per step: 0.0638.. Train loss: 0.1783.. Train accuracy: 0.9370.. Top-3 train accuracy: 0.9950.. Test loss: 0.5910.. Test accuracy: 0.8410.. Top-3 test accuracy: 0.9683\n",
      "Epoch 37/100.. Time per epoch: 25.1080.. Average time per step: 0.0642.. Train loss: 0.1755.. Train accuracy: 0.9393.. Top-3 train accuracy: 0.9949.. Test loss: 0.5457.. Test accuracy: 0.8439.. Top-3 test accuracy: 0.9706\n",
      "Epoch 38/100.. Time per epoch: 24.9298.. Average time per step: 0.0638.. Train loss: 0.1678.. Train accuracy: 0.9417.. Top-3 train accuracy: 0.9955.. Test loss: 0.5865.. Test accuracy: 0.8450.. Top-3 test accuracy: 0.9689\n",
      "Epoch 39/100.. Time per epoch: 24.9929.. Average time per step: 0.0639.. Train loss: 0.1625.. Train accuracy: 0.9433.. Top-3 train accuracy: 0.9955.. Test loss: 0.5874.. Test accuracy: 0.8419.. Top-3 test accuracy: 0.9688\n",
      "Epoch 40/100.. Time per epoch: 24.9632.. Average time per step: 0.0638.. Train loss: 0.1548.. Train accuracy: 0.9459.. Top-3 train accuracy: 0.9962.. Test loss: 0.5741.. Test accuracy: 0.8425.. Top-3 test accuracy: 0.9704\n",
      "Epoch 41/100.. Time per epoch: 24.9623.. Average time per step: 0.0638.. Train loss: 0.1512.. Train accuracy: 0.9472.. Top-3 train accuracy: 0.9965.. Test loss: 0.6020.. Test accuracy: 0.8415.. Top-3 test accuracy: 0.9683\n",
      "Epoch 42/100.. Time per epoch: 25.0070.. Average time per step: 0.0640.. Train loss: 0.1498.. Train accuracy: 0.9481.. Top-3 train accuracy: 0.9965.. Test loss: 0.5676.. Test accuracy: 0.8486.. Top-3 test accuracy: 0.9713\n",
      "Epoch 43/100.. Time per epoch: 24.9706.. Average time per step: 0.0639.. Train loss: 0.1438.. Train accuracy: 0.9496.. Top-3 train accuracy: 0.9968.. Test loss: 0.5822.. Test accuracy: 0.8490.. Top-3 test accuracy: 0.9707\n",
      "Epoch 44/100.. Time per epoch: 24.9036.. Average time per step: 0.0637.. Train loss: 0.1405.. Train accuracy: 0.9513.. Top-3 train accuracy: 0.9967.. Test loss: 0.5766.. Test accuracy: 0.8486.. Top-3 test accuracy: 0.9697\n",
      "Epoch 45/100.. Time per epoch: 25.0014.. Average time per step: 0.0639.. Train loss: 0.1368.. Train accuracy: 0.9528.. Top-3 train accuracy: 0.9972.. Test loss: 0.5962.. Test accuracy: 0.8469.. Top-3 test accuracy: 0.9703\n",
      "Epoch 46/100.. Time per epoch: 24.9887.. Average time per step: 0.0639.. Train loss: 0.1362.. Train accuracy: 0.9517.. Top-3 train accuracy: 0.9971.. Test loss: 0.5826.. Test accuracy: 0.8491.. Top-3 test accuracy: 0.9713\n",
      "Epoch 47/100.. Time per epoch: 24.9850.. Average time per step: 0.0639.. Train loss: 0.1325.. Train accuracy: 0.9538.. Top-3 train accuracy: 0.9972.. Test loss: 0.5757.. Test accuracy: 0.8505.. Top-3 test accuracy: 0.9717\n",
      "Epoch 48/100.. Time per epoch: 25.3707.. Average time per step: 0.0649.. Train loss: 0.1277.. Train accuracy: 0.9566.. Top-3 train accuracy: 0.9973.. Test loss: 0.5984.. Test accuracy: 0.8437.. Top-3 test accuracy: 0.9716\n",
      "Epoch 49/100.. Time per epoch: 25.1389.. Average time per step: 0.0643.. Train loss: 0.1245.. Train accuracy: 0.9570.. Top-3 train accuracy: 0.9973.. Test loss: 0.6192.. Test accuracy: 0.8474.. Top-3 test accuracy: 0.9689\n",
      "Epoch 50/100.. Time per epoch: 25.1840.. Average time per step: 0.0644.. Train loss: 0.1205.. Train accuracy: 0.9583.. Top-3 train accuracy: 0.9979.. Test loss: 0.6052.. Test accuracy: 0.8430.. Top-3 test accuracy: 0.9696\n",
      "Epoch 51/100.. Time per epoch: 25.2480.. Average time per step: 0.0646.. Train loss: 0.1192.. Train accuracy: 0.9587.. Top-3 train accuracy: 0.9978.. Test loss: 0.6074.. Test accuracy: 0.8492.. Top-3 test accuracy: 0.9688\n",
      "Epoch 52/100.. Time per epoch: 25.2910.. Average time per step: 0.0647.. Train loss: 0.1174.. Train accuracy: 0.9587.. Top-3 train accuracy: 0.9977.. Test loss: 0.5926.. Test accuracy: 0.8511.. Top-3 test accuracy: 0.9697\n",
      "Epoch 53/100.. Time per epoch: 25.1977.. Average time per step: 0.0644.. Train loss: 0.1139.. Train accuracy: 0.9600.. Top-3 train accuracy: 0.9978.. Test loss: 0.6198.. Test accuracy: 0.8460.. Top-3 test accuracy: 0.9695\n",
      "Epoch 54/100.. Time per epoch: 25.2504.. Average time per step: 0.0646.. Train loss: 0.1131.. Train accuracy: 0.9608.. Top-3 train accuracy: 0.9977.. Test loss: 0.6257.. Test accuracy: 0.8446.. Top-3 test accuracy: 0.9684\n",
      "Epoch 55/100.. Time per epoch: 25.6454.. Average time per step: 0.0656.. Train loss: 0.1082.. Train accuracy: 0.9625.. Top-3 train accuracy: 0.9978.. Test loss: 0.6214.. Test accuracy: 0.8487.. Top-3 test accuracy: 0.9710\n",
      "Epoch 56/100.. Time per epoch: 25.7317.. Average time per step: 0.0658.. Train loss: 0.1109.. Train accuracy: 0.9613.. Top-3 train accuracy: 0.9982.. Test loss: 0.6158.. Test accuracy: 0.8464.. Top-3 test accuracy: 0.9719\n",
      "Epoch 57/100.. Time per epoch: 25.7459.. Average time per step: 0.0658.. Train loss: 0.1049.. Train accuracy: 0.9638.. Top-3 train accuracy: 0.9982.. Test loss: 0.6187.. Test accuracy: 0.8473.. Top-3 test accuracy: 0.9697\n",
      "Epoch 58/100.. Time per epoch: 25.6894.. Average time per step: 0.0657.. Train loss: 0.1064.. Train accuracy: 0.9633.. Top-3 train accuracy: 0.9981.. Test loss: 0.6152.. Test accuracy: 0.8497.. Top-3 test accuracy: 0.9715\n",
      "Epoch 59/100.. Time per epoch: 25.7970.. Average time per step: 0.0660.. Train loss: 0.1005.. Train accuracy: 0.9641.. Top-3 train accuracy: 0.9984.. Test loss: 0.6281.. Test accuracy: 0.8436.. Top-3 test accuracy: 0.9701\n",
      "Epoch 60/100.. Time per epoch: 25.6835.. Average time per step: 0.0657.. Train loss: 0.0994.. Train accuracy: 0.9654.. Top-3 train accuracy: 0.9981.. Test loss: 0.6249.. Test accuracy: 0.8516.. Top-3 test accuracy: 0.9704\n",
      "Epoch 61/100.. Time per epoch: 25.6254.. Average time per step: 0.0655.. Train loss: 0.0935.. Train accuracy: 0.9668.. Top-3 train accuracy: 0.9984.. Test loss: 0.6486.. Test accuracy: 0.8525.. Top-3 test accuracy: 0.9701\n",
      "Epoch 62/100.. Time per epoch: 25.6922.. Average time per step: 0.0657.. Train loss: 0.0966.. Train accuracy: 0.9666.. Top-3 train accuracy: 0.9985.. Test loss: 0.6228.. Test accuracy: 0.8526.. Top-3 test accuracy: 0.9693\n",
      "Epoch 63/100.. Time per epoch: 25.7218.. Average time per step: 0.0658.. Train loss: 0.0930.. Train accuracy: 0.9684.. Top-3 train accuracy: 0.9986.. Test loss: 0.6678.. Test accuracy: 0.8458.. Top-3 test accuracy: 0.9689\n",
      "Epoch 64/100.. Time per epoch: 25.7628.. Average time per step: 0.0659.. Train loss: 0.0948.. Train accuracy: 0.9676.. Top-3 train accuracy: 0.9982.. Test loss: 0.6573.. Test accuracy: 0.8507.. Top-3 test accuracy: 0.9677\n",
      "Epoch 65/100.. Time per epoch: 25.7151.. Average time per step: 0.0658.. Train loss: 0.0925.. Train accuracy: 0.9682.. Top-3 train accuracy: 0.9986.. Test loss: 0.6351.. Test accuracy: 0.8520.. Top-3 test accuracy: 0.9716\n",
      "Epoch 66/100.. Time per epoch: 25.6193.. Average time per step: 0.0655.. Train loss: 0.0875.. Train accuracy: 0.9699.. Top-3 train accuracy: 0.9985.. Test loss: 0.6511.. Test accuracy: 0.8491.. Top-3 test accuracy: 0.9715\n",
      "Epoch 67/100.. Time per epoch: 24.9405.. Average time per step: 0.0638.. Train loss: 0.0868.. Train accuracy: 0.9695.. Top-3 train accuracy: 0.9987.. Test loss: 0.6487.. Test accuracy: 0.8525.. Top-3 test accuracy: 0.9721\n",
      "Epoch 68/100.. Time per epoch: 24.9615.. Average time per step: 0.0638.. Train loss: 0.0938.. Train accuracy: 0.9674.. Top-3 train accuracy: 0.9985.. Test loss: 0.6461.. Test accuracy: 0.8492.. Top-3 test accuracy: 0.9681\n",
      "Epoch 69/100.. Time per epoch: 24.9491.. Average time per step: 0.0638.. Train loss: 0.0847.. Train accuracy: 0.9710.. Top-3 train accuracy: 0.9987.. Test loss: 0.6402.. Test accuracy: 0.8526.. Top-3 test accuracy: 0.9696\n",
      "Epoch 70/100.. Time per epoch: 24.9763.. Average time per step: 0.0639.. Train loss: 0.0819.. Train accuracy: 0.9717.. Top-3 train accuracy: 0.9989.. Test loss: 0.6506.. Test accuracy: 0.8495.. Top-3 test accuracy: 0.9685\n",
      "Epoch 71/100.. Time per epoch: 25.0945.. Average time per step: 0.0642.. Train loss: 0.0842.. Train accuracy: 0.9709.. Top-3 train accuracy: 0.9990.. Test loss: 0.6629.. Test accuracy: 0.8529.. Top-3 test accuracy: 0.9699\n",
      "Epoch 72/100.. Time per epoch: 24.9095.. Average time per step: 0.0637.. Train loss: 0.0812.. Train accuracy: 0.9720.. Top-3 train accuracy: 0.9989.. Test loss: 0.6392.. Test accuracy: 0.8552.. Top-3 test accuracy: 0.9720\n",
      "Epoch 73/100.. Time per epoch: 24.9105.. Average time per step: 0.0637.. Train loss: 0.0795.. Train accuracy: 0.9720.. Top-3 train accuracy: 0.9991.. Test loss: 0.6605.. Test accuracy: 0.8509.. Top-3 test accuracy: 0.9704\n",
      "Epoch 74/100.. Time per epoch: 25.0891.. Average time per step: 0.0642.. Train loss: 0.0802.. Train accuracy: 0.9721.. Top-3 train accuracy: 0.9990.. Test loss: 0.6847.. Test accuracy: 0.8467.. Top-3 test accuracy: 0.9676\n",
      "Epoch 75/100.. Time per epoch: 24.9808.. Average time per step: 0.0639.. Train loss: 0.0809.. Train accuracy: 0.9727.. Top-3 train accuracy: 0.9988.. Test loss: 0.6784.. Test accuracy: 0.8505.. Top-3 test accuracy: 0.9700\n",
      "Epoch 76/100.. Time per epoch: 24.9689.. Average time per step: 0.0639.. Train loss: 0.0765.. Train accuracy: 0.9739.. Top-3 train accuracy: 0.9989.. Test loss: 0.6782.. Test accuracy: 0.8498.. Top-3 test accuracy: 0.9694\n",
      "Epoch 77/100.. Time per epoch: 25.0902.. Average time per step: 0.0642.. Train loss: 0.0748.. Train accuracy: 0.9737.. Top-3 train accuracy: 0.9992.. Test loss: 0.6658.. Test accuracy: 0.8519.. Top-3 test accuracy: 0.9724\n",
      "Epoch 78/100.. Time per epoch: 25.0309.. Average time per step: 0.0640.. Train loss: 0.0786.. Train accuracy: 0.9732.. Top-3 train accuracy: 0.9988.. Test loss: 0.6643.. Test accuracy: 0.8527.. Top-3 test accuracy: 0.9695\n",
      "Epoch 79/100.. Time per epoch: 24.9545.. Average time per step: 0.0638.. Train loss: 0.0708.. Train accuracy: 0.9757.. Top-3 train accuracy: 0.9993.. Test loss: 0.6654.. Test accuracy: 0.8558.. Top-3 test accuracy: 0.9710\n",
      "Epoch 80/100.. Time per epoch: 25.0356.. Average time per step: 0.0640.. Train loss: 0.0757.. Train accuracy: 0.9736.. Top-3 train accuracy: 0.9990.. Test loss: 0.6494.. Test accuracy: 0.8534.. Top-3 test accuracy: 0.9711\n",
      "Epoch 81/100.. Time per epoch: 24.9606.. Average time per step: 0.0638.. Train loss: 0.0707.. Train accuracy: 0.9759.. Top-3 train accuracy: 0.9991.. Test loss: 0.6864.. Test accuracy: 0.8506.. Top-3 test accuracy: 0.9717\n",
      "Epoch 82/100.. Time per epoch: 25.1393.. Average time per step: 0.0643.. Train loss: 0.0708.. Train accuracy: 0.9755.. Top-3 train accuracy: 0.9990.. Test loss: 0.6698.. Test accuracy: 0.8524.. Top-3 test accuracy: 0.9693\n",
      "Epoch 83/100.. Time per epoch: 24.9794.. Average time per step: 0.0639.. Train loss: 0.0689.. Train accuracy: 0.9756.. Top-3 train accuracy: 0.9991.. Test loss: 0.6826.. Test accuracy: 0.8504.. Top-3 test accuracy: 0.9711\n",
      "Epoch 84/100.. Time per epoch: 24.9459.. Average time per step: 0.0638.. Train loss: 0.0705.. Train accuracy: 0.9752.. Top-3 train accuracy: 0.9992.. Test loss: 0.6650.. Test accuracy: 0.8541.. Top-3 test accuracy: 0.9705\n",
      "Epoch 85/100.. Time per epoch: 25.0073.. Average time per step: 0.0640.. Train loss: 0.0686.. Train accuracy: 0.9756.. Top-3 train accuracy: 0.9993.. Test loss: 0.6922.. Test accuracy: 0.8473.. Top-3 test accuracy: 0.9694\n",
      "Epoch 86/100.. Time per epoch: 24.9635.. Average time per step: 0.0638.. Train loss: 0.0656.. Train accuracy: 0.9772.. Top-3 train accuracy: 0.9993.. Test loss: 0.6686.. Test accuracy: 0.8557.. Top-3 test accuracy: 0.9727\n",
      "Epoch 87/100.. Time per epoch: 24.9003.. Average time per step: 0.0637.. Train loss: 0.0697.. Train accuracy: 0.9763.. Top-3 train accuracy: 0.9993.. Test loss: 0.6724.. Test accuracy: 0.8534.. Top-3 test accuracy: 0.9712\n",
      "Epoch 88/100.. Time per epoch: 24.9206.. Average time per step: 0.0637.. Train loss: 0.0640.. Train accuracy: 0.9776.. Top-3 train accuracy: 0.9993.. Test loss: 0.6882.. Test accuracy: 0.8544.. Top-3 test accuracy: 0.9717\n",
      "Epoch 89/100.. Time per epoch: 24.9898.. Average time per step: 0.0639.. Train loss: 0.0664.. Train accuracy: 0.9771.. Top-3 train accuracy: 0.9992.. Test loss: 0.6799.. Test accuracy: 0.8524.. Top-3 test accuracy: 0.9701\n",
      "Epoch 90/100.. Time per epoch: 24.9307.. Average time per step: 0.0638.. Train loss: 0.0660.. Train accuracy: 0.9767.. Top-3 train accuracy: 0.9991.. Test loss: 0.7133.. Test accuracy: 0.8446.. Top-3 test accuracy: 0.9702\n",
      "Epoch 91/100.. Time per epoch: 24.9342.. Average time per step: 0.0638.. Train loss: 0.0647.. Train accuracy: 0.9775.. Top-3 train accuracy: 0.9992.. Test loss: 0.6820.. Test accuracy: 0.8504.. Top-3 test accuracy: 0.9688\n",
      "Epoch 92/100.. Time per epoch: 24.8989.. Average time per step: 0.0637.. Train loss: 0.0645.. Train accuracy: 0.9777.. Top-3 train accuracy: 0.9993.. Test loss: 0.6653.. Test accuracy: 0.8548.. Top-3 test accuracy: 0.9688\n",
      "Epoch 93/100.. Time per epoch: 25.0700.. Average time per step: 0.0641.. Train loss: 0.0614.. Train accuracy: 0.9785.. Top-3 train accuracy: 0.9993.. Test loss: 0.7087.. Test accuracy: 0.8441.. Top-3 test accuracy: 0.9701\n",
      "Epoch 94/100.. Time per epoch: 24.8299.. Average time per step: 0.0635.. Train loss: 0.0645.. Train accuracy: 0.9772.. Top-3 train accuracy: 0.9990.. Test loss: 0.6823.. Test accuracy: 0.8538.. Top-3 test accuracy: 0.9709\n",
      "Epoch 95/100.. Time per epoch: 24.8429.. Average time per step: 0.0635.. Train loss: 0.0620.. Train accuracy: 0.9783.. Top-3 train accuracy: 0.9993.. Test loss: 0.6865.. Test accuracy: 0.8484.. Top-3 test accuracy: 0.9719\n",
      "Epoch 96/100.. Time per epoch: 24.9001.. Average time per step: 0.0637.. Train loss: 0.0584.. Train accuracy: 0.9807.. Top-3 train accuracy: 0.9993.. Test loss: 0.7094.. Test accuracy: 0.8519.. Top-3 test accuracy: 0.9689\n",
      "Epoch 97/100.. Time per epoch: 24.8832.. Average time per step: 0.0636.. Train loss: 0.0590.. Train accuracy: 0.9789.. Top-3 train accuracy: 0.9993.. Test loss: 0.7057.. Test accuracy: 0.8523.. Top-3 test accuracy: 0.9712\n",
      "Epoch 98/100.. Time per epoch: 24.9052.. Average time per step: 0.0637.. Train loss: 0.0605.. Train accuracy: 0.9788.. Top-3 train accuracy: 0.9992.. Test loss: 0.6577.. Test accuracy: 0.8553.. Top-3 test accuracy: 0.9726\n",
      "Epoch 99/100.. Time per epoch: 24.7946.. Average time per step: 0.0634.. Train loss: 0.0538.. Train accuracy: 0.9816.. Top-3 train accuracy: 0.9995.. Test loss: 0.6997.. Test accuracy: 0.8540.. Top-3 test accuracy: 0.9721\n",
      "Epoch 100/100.. Time per epoch: 24.9208.. Average time per step: 0.0637.. Train loss: 0.0578.. Train accuracy: 0.9807.. Top-3 train accuracy: 0.9994.. Test loss: 0.6998.. Test accuracy: 0.8527.. Top-3 test accuracy: 0.9709\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_MobileNet_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
